package com.chance.cc.crawler.development.bootstrap.diaoyu;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;

public class DiaoYu {
    public static void main(String[] args) {
        String domain = "diaoyu";
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .httpUrl("http://bbs.diaoyu.com/subject-list-1-1.html?orderby=reply")
                .recordKey("http://bbs.diaoyu.com/subject-list-1-1.html?orderby=reply")
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*3,null))
                .releaseTime(System.currentTimeMillis())
                .build();
        requestRecord.tagsCreator().bizTags().addCustomKV("domain",domain);
        requestRecord.tagsCreator().bizTags().addDomain(domain);

        DevCrawlerController.builder()
                .triggerInfo(domain,domain,System.currentTimeMillis(),domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))
                .consoleResultPipeline()
                .fileResultPipeline("D:\\chance\\log\\diaoyu\\dy.log",false)
                .requestRecord(requestRecord)
                .build()
                .start();





    }
}
